package util;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;

import util.ViewUtils.CharType;

/**
 * TermLabeler is a tool to label out the terms in the text. Note this class
 * is not synchronized and not thread-safe. For multi-thread cases, please
 * use a pool like ThreadLocal.
 * @author zhoumin
 */
@SuppressWarnings("unchecked")
public class TermLabeler {
    private static final String regex = "&_&";    // the symbol will be replaced with detailDecor
    private static StringSet stopWords;    // the set of stop words
    private static String specialPunctuations = " !@#$&-_/."; // sentence dividers
    
    /**
     * The decoration for a labeled term
     * @author fangqi
     */
    private static class Decor {                            
        public final String prefix;    // prefix will be inserted before the term
        public final String suffix;    // suffix will be inserted after the term
        public Decor(String prefix, String suffix) {
            this.prefix = prefix;
            this.suffix = suffix;
        }// constructor
    }
    
    /**
     * The result of labeling. 
     * It contains the labeled string and matched term indexes.
     * @author jieme
     */
    public class TermMatchInfo {
        public final int pos;
        public final int length;
        public TermMatchInfo (int pos, int length){
            this.pos = pos;
            this.length = length;
        }
    }
    
    public class LabelResult {
        public String labeledText;
        public final ArrayList<TermMatchInfo>[] termMatchInfos;
        public LabelResult(int len) {
            termMatchInfos = new ArrayList[len];
            for ( int i=0; i<len; i++ ) {
                termMatchInfos[i] = new ArrayList<TermMatchInfo>();
            }
        }
    }
    
    private final ArrayList<Decor> decors = new ArrayList<Decor>();          // the list of Decor  
    private final ArrayList<Decor> detailDecors = new ArrayList<Decor>();    // the list of detail decors
    private final ArrayList<Integer> htmlTagBegin = new ArrayList<Integer>();  // records the position of htmlTagBegin symbol    
    private final ArrayList<Integer> htmlTagEnd = new ArrayList<Integer>();    // records the position of htmlTagEnd symbol
    
    /*
     * records the matching states of every char of the text,
     * =-1    no term matched with,
     * >-1    the id of QueryTerm matched with 
     */    
    private byte[] map;    
    private byte[] termIndex, mapMatchLen;    
    // termIndex,  records which QueryTerm this term belong to
    // mapMatchLen ,records the matching length of every char of the text,
    //             -1  no term matched with,
    //             >0  the length of the term which match with the string beginning with this char,
    //             =0  this char is not the first char of the matched term
    
    private int threshold = 2;    //  the length threshold for output the labeled terms       
    
    // record the result which contains the label string 
    // and matched term indexes. Assign initial value null first. 
    private LabelResult result = null;
    
    /**
     * @param path   file path
     * @throws IOException
     */
    public static void setStopWordPath(String path) throws IOException {
        stopWords = new StringSet(path);
    }
    
    /**
     * @return
     */
    public int getThreshold() {
        return threshold;
    }

    /**
     * @param thrhld  
     */
    public void setThreshold(int threshold) {
        this.threshold = threshold;
    }

    /**
     * Add a decoration pair to the list of decors. The decoration pairs are used to label terms.
     * @param prefix Prefix will be inserted before the term
     * @param suffix Suffix will be inserted after the term
     */
    public void addDecor(String prefix, String suffix) {
        decors.add(new Decor(prefix, suffix));
    }
    
    /**
     * Add a decoration pair to the list of detail decors. The decoration pairs are used to label terms. 
     * @param prefix prefix will be inserted before the term
     * @param suffix Suffix will be inserted after the term
     */
    public void addDetailDecor(String prefix, String suffix) {
        detailDecors.add(new Decor(prefix, suffix));
    }

    /**
     * Get the labeled version of the text, using terms and queryTerms
     * @param text   the text to be labeled
     * @param terms  the parsed terms 
     * @param queryTerms  the init terms, before pasrsing
     * @param skipHTMLFlag whether need to skip HTML flag
     * @return
     */
    public String twoPhaseLabel(String text, String[] terms, String[] queryTerms, 
            boolean skipHTMLFlag) {
        return twoPhaseLabel(text, terms, queryTerms, skipHTMLFlag, false);
    }
    
    /**
     * Get the labeled version of the text, using terms and queryTerms
     * @param text   the text to be labeled
     * @param terms  the parsed terms 
     * @param queryTerms  the init terms, before pasrsing
     * @param skipHTMLFlag whether need to skip HTML flag
     * @param matchAllFlag whether labeled when not all terms are matched, true:not label, false:label
     * @return
     */
    public String twoPhaseLabel(String text, String[] terms, String[] queryTerms, 
            boolean skipHTMLFlag, boolean matchAllFlag) {
        try {
            int textLength = text.length();
            //尝试Label操作，如果不成功则直接输出原来的text
            if (!labelProcess(text, terms, queryTerms, skipHTMLFlag, matchAllFlag))
                return text;
            
            // output
            int i;
            int[] t_index = new int[queryTerms.length];
            for(i = 0; i < t_index.length; i++)
                t_index[i] = 1;

            StringBuilder buffer = new StringBuilder();
            i = 0;
            while(i < textLength) {
                if (mapMatchLen[i] >= threshold || 
                   (mapMatchLen[i] > 0 &&
                           ((queryTerms[map[i]].length() < threshold) || 
                             (i > 0 && map[i - 1] >= 0) || 
                             ((i + mapMatchLen[i]) < textLength && map[i + mapMatchLen[i]] >= 0)
                                                                         ))){
                    // 给text里面每一个匹配的term作标记，格式为"匹配的Term编号_第几次匹配"
                    String replacement = new String((map[i] + 1) + "_" + (t_index[map[i]]++));
                    if (decors.size() > 0)
                        buffer.append(decors.get(map[i]%decors.size()).prefix);
                    if (detailDecors.size() > 0) {
                        String ss = detailDecors.get(map[i] % detailDecors.size()).prefix;
                        buffer.append(ss.replaceAll(regex, replacement));
                    }// if

                    buffer.append(text.substring(i, i + mapMatchLen[i]));
                    
                    if (detailDecors.size() > 0) {
                        String ss = detailDecors.get(map[i]%detailDecors.size()).suffix;
                        buffer.append(ss.replaceAll(regex, replacement));
                    }// if
                    if (decors.size() > 0)
                        buffer.append(decors.get(map[i] % decors.size()).suffix);
                    i = i + mapMatchLen[i];
                } else {
                    buffer.append(text.charAt(i));
                    ++i;
                }// else
            }// while
            return buffer.toString();
        } catch (Exception e) {
            System.err.println("Exception in TermLabeler, text=" + 
                    text + ", terms=" + ViewUtils.printArray(terms) + 
                    ", queryTerms=" + ViewUtils.printArray(queryTerms) + 
                    ", skipHTMLFlag=" + skipHTMLFlag);
            return text;
        }// try-catch
    }
    
    private boolean labelProcess(String text, String[] terms, String[] queryTerms, 
            boolean skipHTMLFlag, boolean matchAllFlag) {
        int textLength = text.length();
        if (textLength == 0 || queryTerms.length == 0) {
            // empty text or empty queryTerms
            return false;
        }// if
        
        int i, j;
        // data initialization, for consideration of efficiency, we reallocate the
        // array only when needed.
        if (map == null || textLength > map.length)
            map = new byte[textLength];
        if (mapMatchLen == null || textLength > mapMatchLen.length)
            mapMatchLen = new byte[textLength];
        Arrays.fill(map, (byte)(-1));
        Arrays.fill(mapMatchLen, (byte)(-1));
        if (termIndex == null || termIndex.length < queryTerms.length)
            termIndex = new byte[queryTerms.length];
        for(i = 0; i < queryTerms.length; i++)
            termIndex[i] = (byte)i;

        if (skipHTMLFlag){
            // if skipHTMLFlag is true , record the position of '<' and '>'
            htmlTagBegin.clear();
            htmlTagEnd.clear();
            
            i = 0;
            while ((i = text.indexOf("<",i)) != -1) {
                htmlTagBegin.add(i);
                i = text.indexOf(">", i);
                if(i != -1) 
                    htmlTagEnd.add(i);
                else {
                    // in this case , we just break it
                    htmlTagBegin.remove(htmlTagBegin.size() - 1);
                    // LOG.info("Error happened: no matched < and >");
                    break;
                }// else
                ++i;
            }// while
        }// if

        // label without parsed terms, only use queryTerms, do NOT skip stop words
        doLabel(text, queryTerms, skipHTMLFlag, false, false);
        
        // check that whether all the queryTerms are found in the text
        HashSet<Integer> foundQueryTerms = new HashSet<Integer>();
        for(i = 0; i < textLength; i++)
            if(map[i] > -1) foundQueryTerms.add((int)map[i]);
        
        if (foundQueryTerms.size() < queryTerms.length) {
            if (termIndex == null || termIndex.length < terms.length)
                termIndex = new byte[terms.length];
            // update termIndex for new term serial
            List<String> termList = new ArrayList<String>();
            boolean[] appeared = new boolean[terms.length];
            for (i = 0; i < terms.length; i++) {
                appeared[i] = false;
                String termStr = terms[i].toLowerCase();
                for (int foundIndex : foundQueryTerms) {
                    String queryTerm = queryTerms[foundIndex];
                    if (queryTerm.toLowerCase().indexOf(termStr) >= 0) {
                        appeared[i] = true;
                        break;
                    }
                }
            }
            int termCount = 0;
            for (i = 0; i < terms.length; i++) {
                if (!appeared[i]) {
                    termList.add(terms[i]);
                    // if a term does NOT belong to any queryTerm, we simply denote that 
                    // it belongs to the first queryTerm.
                    // notice: queryTerm: 色，戒, terms: 色戒
                    termIndex[termCount] = 0;
                    for (j = 0; j < queryTerms.length; j++) {
                        if (queryTerms[j].toLowerCase().indexOf(terms[i].toLowerCase()) >= 0) {
                            termIndex[termCount] = (byte)j;
                            break;
                        }
                    }
                    termCount++;
                }
            }
            // label the text again using parsed terms
            // if not all terms are matched and matchAllFlag is seted, then do nothing
            String[] parsedTerms = new String[termList.size()];
            termList.toArray(parsedTerms);
            // skip stop words
            if (!doLabel(text, parsedTerms, skipHTMLFlag, true, true) 
                    && matchAllFlag)
                return false;
        }
        
        // merge two adjacent matched terms while they appear in the same queryTerm adjacently
        i = 0;
        int lastMapIndex = -1; // the index that last matched term starts
        while (i < textLength) {
            if (mapMatchLen[i] > 0){
                if (lastMapIndex < 0){ // the first matched term
                    lastMapIndex = i;
                    i += mapMatchLen[i];
                } else {
                    if(map[i] == map[lastMapIndex]) { 
                        //in the same queryTerm
                        if (((i + mapMatchLen[i] - lastMapIndex) <= queryTerms[map[i]].length()) &&
                            (queryTerms[map[i]].toLowerCase().indexOf(
                             text.substring(lastMapIndex, i + mapMatchLen[i]).toLowerCase()) >= 0) ){
                            //adjacent in queryTerm, merge them
                            i += mapMatchLen[i];
                            Arrays.fill(map, lastMapIndex + 1, i, (byte)map[lastMapIndex]);
                            mapMatchLen[lastMapIndex] = (byte)(i - lastMapIndex);
                            Arrays.fill(mapMatchLen, lastMapIndex + 1, i, (byte)0);
                        } else 
                        if ((i - (lastMapIndex + mapMatchLen[lastMapIndex])) == 1 && 
                                isSpecialPunctuation(text.charAt(i - 1))) {
                            // punctuation in middle
                            map[i - 1] = map[lastMapIndex];
                            mapMatchLen[i - 1] = 0;
                            mapMatchLen[lastMapIndex]++;
                            lastMapIndex = i;
                            i += mapMatchLen[i];
                        } else {
                            // not adjacent in queryTerm, do nothing
                            lastMapIndex = i;
                            i += mapMatchLen[i];
                        }// else
                    } else { 
                        // not in the same queryTerm
                        if( ((i + mapMatchLen[i] - lastMapIndex) <= queryTerms[map[i]].length()) &&
                                (queryTerms[map[i]].toLowerCase().indexOf(
                                 text.substring(lastMapIndex, i + mapMatchLen[i]).toLowerCase())>=0) ){
                                //adjacent in queryTerm map[i], merge them
                                //i+=mapMatchLen[i];
                                Arrays.fill(map, lastMapIndex + 1, i + mapMatchLen[i], (byte)map[i]);
                                i += mapMatchLen[i];
                                mapMatchLen[lastMapIndex] = (byte)(i - lastMapIndex);
                                Arrays.fill(mapMatchLen, lastMapIndex + 1, i, (byte)0);
                            } else
                            if (((i + mapMatchLen[i] - lastMapIndex) <= queryTerms[map[lastMapIndex]].length()) &&
                                (queryTerms[map[lastMapIndex]].toLowerCase().indexOf(
                                 text.substring(lastMapIndex, i + mapMatchLen[i]).toLowerCase()) >= 0) ){
                                //adjacent in queryTerm map[lastMapIndex], merge them
                                i += mapMatchLen[i];
                                Arrays.fill(map, lastMapIndex + 1, i, (byte)map[lastMapIndex]);
                                mapMatchLen[lastMapIndex]=(byte)(i - lastMapIndex);
                                Arrays.fill(mapMatchLen, lastMapIndex + 1, i, (byte)0);
                            } else
                            if ((i - (lastMapIndex + mapMatchLen[lastMapIndex])) == 1 && 
                                    isSpecialPunctuation(text.charAt(i - 1))) {
                                // punctuation in middle
                                map[i - 1] = map[lastMapIndex];
                                mapMatchLen[i - 1] = 0;
                                mapMatchLen[lastMapIndex]++;
                                lastMapIndex = i;
                                i += mapMatchLen[i];
                            }else {
                                // not adjacent in queryTerm, do nothing
                                lastMapIndex = i;
                                i += mapMatchLen[i];
                            }// else
                    }// else
                }// else
            } else {
                i++;
            }// else
        }// while
        return true;
    }

    /**
     * Get the labeled version of the text
     * @param text the text to be labeled
     * @param terms the terms to search
     * @param skipHTMLFlag  whether need to skip HTML flag
     * @param skipStopword  whether need to skip stop word
     * @return if all terms are matched return true, else false.
     */    
    private boolean doLabel(String text, String[] terms, 
            boolean skipHTMLFlag, boolean skipStopword, boolean usingQueryTerm) {
        int i, j;
        int textLength = text.length();
        // matchedRecord[i] records the num of terms which are found to match with 
        // the string from text[0] to text[i-1] before this label function starts        
        int[] matchedRecord = new int [textLength + 1];    
        // maxValue[i] records the max matched length from text[0] to text[i-1];  
        int[] maxValue = new int[textLength + 1];   
        // selectTerm[i] records the last selected term in the optimal solution of 
        // labeling the string from text[0] to text[i-1]
        int[] selectTerm = new int[textLength + 1];  

        // termMatched[i] records whether terms[i] is matched or not
        boolean[] termMatched = new boolean[terms.length];

        // point to the next tag
        int htmlTagIndex = 0;
        // mark whether curren char is in a tag
        boolean inTag = false;
        
        // data initialization
        maxValue[0] = 0;        
        selectTerm[0] = -1;
        matchedRecord[0] = 0;
        for (i = 0; i < textLength; i++) {
            matchedRecord[i + 1] = matchedRecord[i];
            if (map[i] > -1) 
                matchedRecord[i + 1]++;            
        }// for 
        for (i = 0; i < terms.length; i++)
            termMatched[i] = false;
       
        // dynamic programming 
        for (i = 1; i <= textLength; i++) {
            selectTerm[i] = -1;
            maxValue[i] = maxValue[i - 1];
            if (map[i - 1] > -1) {
                maxValue[i]++;
                continue;
            }// if          

            if ( skipHTMLFlag && htmlTagIndex < htmlTagBegin.size() ) 
                    if (htmlTagBegin.get(htmlTagIndex) == (i-1)) {
                        inTag = true;
                    } else if ((i - 1) == htmlTagEnd.get(htmlTagIndex)) {
                        inTag = false;
                        htmlTagIndex++;
                        continue;
                    } // else - if
            if (inTag)
                continue;
            
            for (j = 0; j < terms.length; j++) {
                if (i >= terms[j].length() && terms[j].length() > 0)
                    if (matchedRecord[i] == matchedRecord[i - terms[j].length()]) {
                        // string from text[i-terms[j].length()-1] to text[i-1] doesn't belong to any terms
                        if (terms[j].equalsIgnoreCase(text.substring(i - terms[j].length(), i))) {
                            // terms[j] match with string from text[i-terms[j].length()-1] to text[i-1]
                            // western terms only match whole words
                            int prefixPos = i - terms[j].length() - 1;
                            int suffixPos = i;

                            if( prefixPos >= 0 &&
                                ((isLetter(text.charAt(prefixPos)) && isLetter(text.charAt(prefixPos + 1))) ||
                                 (isDigit(text.charAt(prefixPos)) && isDigit(text.charAt(prefixPos + 1))))) {
                                // this term has a prefix
                                continue;
                            }// if

                            if( suffixPos < textLength &&
                                ((isLetter(text.charAt(suffixPos)) && isLetter(text.charAt(i - 1))) ||
                                 (isDigit(text.charAt(suffixPos)) && isDigit(text.charAt(i - 1))))) {
                                // this term has a suffix
                                continue;
                            }// if                                       

                            
                            if (maxValue[i - terms[j].length()] + terms[j].length() > maxValue[i]) {
                                // it is better using terms[j]
                                maxValue[i] = maxValue[i - terms[j].length()] + terms[j].length();
                                selectTerm[i] = j;
                            }// if
                        }// if
                    }// if
            }// for
        }// for
        
        // update map and mapMatchLen ; put the optimal solution into practical effect
        i = textLength;
        int tskip = 0; // records the num of term found in the text which is a stopWord and not closed to any other terms
        while (i > 0) {
            while (selectTerm[i] == -1 && i > 0) i--;
            if (i == 0) break;
            
            int l = terms[selectTerm[i]].length();
            boolean ok = true;
            
            if (skipStopword && l == 1 && stopWords != null && stopWords.contains(terms[selectTerm[i]]))  // the char of text[i-1] is a stop word
                if (i == 1 || (map[i - 2] == -1 && selectTerm[i - 1] == -1))   // the char of text[i-2] doesn't belong to any terms
                    if (i == textLength || map[i] == -1) {
                        ok = false;
                        tskip++;
                    }// if                  

            if (ok) {
                Arrays.fill(map, i - l, i, (byte)termIndex[selectTerm[i]]);
                mapMatchLen[i - l] = (byte)l;
                if (l > 1)
                    Arrays.fill(mapMatchLen, i - l + 1, i, (byte)0);
                termMatched[selectTerm[i]] = true;
                if (result != null && usingQueryTerm) {
                    /*
                     * if result has been initialized and we are using QueryTerms, 
                     * we have to record matching positions which are used by
                     * other services: SNAPSHOT
                     */ 
                    TermMatchInfo m = new TermMatchInfo(i - 1, l);
                    result.termMatchInfos[termIndex[selectTerm[i]]].add(m);
                } // if
            }// if
            i = i - l;
        }// while
        
        // all matched word are stopwords
        if (skipStopword && tskip > 0 && maxValue[textLength] == tskip) {
            i = textLength;
            while (i > 0) {
                while (i > 0 && selectTerm[i] == -1) i--;
                if (i == 0) break;            
                int l = terms[selectTerm[i]].length();
                if (map[i - 1] == -1) {
                    Arrays.fill(map, i - l, i, (byte)termIndex[selectTerm[i]]);
                    mapMatchLen[i - l] = (byte)l;
                    if (l > 1)
                        Arrays.fill(mapMatchLen, i - l + 1, i, (byte)0);
                }//if
                i = i - l;          
            }//while
        }//if
        
        for (i = 0; i < terms.length; i++)
            if (!termMatched[i])
                return false;
        return true;
    }
       
    /**
     * Get the labeled version of the text, using terms and queryTerms
     * @param items  to be labeled contains title, text, url
     * @param terms  the parsed terms 
     * @param queryTerms  the init terms, before pasrsing
     * @param skipHTMLFlag whether need to skip HTML flag
     * @return
     */
    public String[] twoPhaseLabel(String[] items, String[] terms, 
               String[] queryTerms, boolean skipHTMLFlag) {
        try {

            if (items.length == 0 || queryTerms.length == 0) {
                // empty items or empty query tems
                return items;
            } // if
                
            // record index of every item in text
            int[] itemIndex = new int[items.length];
            StringBuilder textBuffer = new StringBuilder();
            itemIndex[0] = 0;
            int k = 0;
            while (true) {
                textBuffer.append(items[k]);
                // this char plays a rollor of a "separator" between items.
                // use any char that most probably not appear in a query, 
                // so that the joint of two items is not matched by a term.
                textBuffer.append('\t');
                k++;
                if (k < items.length)
                    itemIndex[k] = itemIndex[k - 1] + items[k - 1].length() + 1;
                else break;
            } // while
            
            if (textBuffer.length() == 0 ) {
                // all items are empty
                return items;
            }// if
            
            String text = textBuffer.toString();
            
            // int textLength = text.length();
            // do NOT need to match all flag in this case
            labelProcess(text, terms, queryTerms, skipHTMLFlag, false);
            // output
            int i, j;
            k = 0;
            String[] labelResult = new String[items.length];
            int[] tIndex = new int[queryTerms.length];
            while (k < items.length){
                StringBuilder buffer = new StringBuilder();
                i = itemIndex[k];
                for(j = 0; j < tIndex.length; j++)
                    tIndex[j] = 1;
                while (i < (itemIndex[k] + items[k].length())) {
                    if ((mapMatchLen[i] >= threshold &&
                            (i + mapMatchLen[i]) <= (itemIndex[k] + items[k].length())) || 
                        (mapMatchLen[i] > 0 &&
                                ((queryTerms[map[i]].length() < threshold) ||
                                        (i > itemIndex[k] && map[i - 1] >= 0) ||
                                        ((i + mapMatchLen[i]) <= (itemIndex[k] + items[k].length())
                                                && map[i + mapMatchLen[i]] >= 0))) ) {
                        String replacement = new String((map[i] + 1) + "_" + (tIndex[map[i]]++));
                        if (decors.size() > 0)
                            buffer.append(decors.get(k % decors.size()).prefix);
                        if (detailDecors.size()>0) {
                            String ss = detailDecors.get(k % detailDecors.size()).prefix;
                            buffer.append(ss.replaceAll(regex, replacement));
                        }// if

                        buffer.append(text.substring(i, i + mapMatchLen[i]));
                             
                        if (detailDecors.size() > 0) {
                            String ss = detailDecors.get(k % detailDecors.size()).suffix;
                            buffer.append(ss.replaceAll(regex, replacement));
                        }// if
                        if (decors.size() > 0)
                             buffer.append(decors.get(k % decors.size()).suffix);
                        i = i + mapMatchLen[i];
                    } else {
                         buffer.append(text.charAt(i));
                         ++i;
                    }// else
                } // while
                labelResult[k++] = buffer.toString();              
            } // while
            
            return labelResult;
        } catch (Exception e) {
            System.err.println("Exception in TermLabeler, text=" + 
                           ViewUtils.printArray(items) + ", terms=" + 
                           ViewUtils.printArray(terms) + ", queryTerms=" + 
                           ViewUtils.printArray(queryTerms) + ", skipHTMLFlag=" +
                           skipHTMLFlag);
            return items;
        }// try-catch
    }

    /**
     * Get the labeled version of the text, using terms and queryTerms
     * @param text   the text to be labeled
     * @param terms  the parsed terms 
     * @param queryTerms  the init terms, before pasrsing
     * @param skipHTMLFlag whether need to skip HTML flag
     * @return label result
     */
    public LabelResult twoPhaseLabelEx(String text, String[] terms, 
            String[] queryTerms, boolean skipHTMLFlag) {
        result = new LabelResult(queryTerms.length);
        result.labeledText = twoPhaseLabel(text, terms, queryTerms, skipHTMLFlag, false);

        Comparator<TermMatchInfo> comparator = new Comparator<TermMatchInfo>() {
            public int compare(TermMatchInfo t1, TermMatchInfo t2) {
                return t1.pos - t2.pos;
            } //compare
        };//compare object

        for ( int i=0; i < queryTerms.length; i++ ) {
            Collections.sort(result.termMatchInfos[i], comparator);
        }  // for
        return result;
    }
    
    /**
     * @param ch
     * @return
     */
    boolean isLetter(char ch) {
        return ViewUtils.getCharType(ch) == CharType.LETTER;
    }
    
    /**
     * @param ch
     * @return
     */
    boolean isDigit(char ch) {
        return ViewUtils.getCharType(ch) == CharType.NUM;     
    }
    
    boolean isSpecialPunctuation(char ch) {
        if ( specialPunctuations.indexOf(ch) != -1 )
            return true;
        return false;
    }
    
}
